# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
from itemadapter import ItemAdapter


# 如果想使用管道的话，那么就必须在settings.py中来开启管道
class DangdangwangPipeline:

    # 在爬虫文件开始之前就执行的方法
    def open_spider(self, spider):
        self.f = open("book.json", "w", encoding="utf-8")
        print("\n=================== open_spider ===================\n")

    # 这里的item就是yield后面的book对象
    def process_item(self, item, spider):
        # 这种写法不推荐，对文件操纵过于频繁
        # with open("book.json", "a", encoding="utf-8") as f:
        #     f.write(str(item) + "\n")

        # 这种写法推荐，对文件的操作只进行一次
        self.f.write(str(item) + "\n")
        return item

    # 爬虫文件结束之后执行的方法
    def close_spider(self, spider):
        self.f.close()
        print("\n=================== close_spider ===================\n")


import urllib.request


# 多个管道开启
# 1 定义管道类
# 2 在settings.py中开启管道
class DangdangwangPipeline2:
    def process_item(self, item, spider):

        url = "http:" + item.get("src")
        filename = "./books/" + item.get("name") + ".jpg"

        urllib.request.urlretrieve(url=url, filename=filename)

        return item
